#! /usr/bin/python2.5
# This file is part of SyncStar, free software released under GPL v2.
# Google AppEngine Python 2.5 source.

import cPickle
import zlib

# `import msgpack' would be more faster, but that's not available on the
# AppEngine production servers.
import aalib.msgpack_pure as msgpack

# uncompressed JSON: 101852 bytes
# 9: 15729
# 8: 15729
# 7: 15761
# 6: 15832
# 5: 15956
# 4: 16283
# 3: 17110
# 2: 17445
# 1: 17791
# 0: 61811
# TODO: Measure compression and decompression speed as well.
COMPRESS_EFFORT = 6  # Between 0 and 9 (slowest and smallest).

# This affects serialize_messsages() only. parse_messages() always autodetects
# Pickle or msgpack.
USE_MSGPACK = False

def unicode_to_utf8(s):
  if isinstance(s, str):
    return s
  elif isinstance(s, unicode):
    return s.encode('UTF-8')
  else:
    raise TypeError(type(s))


def utf8_to_unicode(s):
  if isinstance(s, unicode):
    return s
  elif isinstance(s, str):
    return s.decode('utf-8')
  else:
    raise TypeError(type(s))


def _convert_to_serializable(message):
  if not isinstance(message, dict):
    raise TypeError
  address = unicode_to_utf8(message.get('address', ''))
  conversation = unicode_to_utf8(message.get('conversation', ''))
  if conversation == address:
    conversation = ''
  return [
      address,
      unicode_to_utf8(message.get('body', '')),
      int(message.get('date', 0)),
      int(message.get('type', 1)),
      unicode_to_utf8(message.get('service_center', '')),
      int(message.get('error_code', 0)),
      int(message.get('locked', 0)),
      int(message.get('read', 0)),
      int(message.get('seen', 0)),
      int(message.get('status', -1)),
      int(message.get('protocol', 0)),
      unicode_to_utf8(message.get('phone', 'ap')),
      conversation,
      bool(message.get('deleted', False)),
  ]

def _convert_from_serializable(sermsg):
  # We accept a tuple because msgpack converts lists to tuples.
  if not isinstance(sermsg, (list, tuple)):
    raise TypeError
  assert len(sermsg) == 14
  return {
      'address': utf8_to_unicode(sermsg[0]),
      'body': utf8_to_unicode(sermsg[1]),
      'date': sermsg[2],
      'type': sermsg[3],
      'service_center': utf8_to_unicode(sermsg[4]),
      'error_code': sermsg[5],
      'locked': sermsg[6],
      'read': sermsg[7],
      'seen': sermsg[8],
      'status': sermsg[9],
      'protocol': sermsg[10],
      'phone': utf8_to_unicode(sermsg[11]),
      'conversation': utf8_to_unicode(sermsg[12]),
      'deleted': sermsg[13],
  }


def fix_message(message):
  # TODO: Speed it up.
  return _convert_from_serializable(_convert_to_serializable(message))


def get_message_key(message):
  return {
      'address': utf8_to_unicode(message.get('address') or ''),
      'body': utf8_to_unicode(message.get('body') or ''),
      'date': int(message.get('date', 0)),
      'type': int(message.get('type', 1)),
  }


def serialize_messages(messages):
  """Returns a byte string (msgsdata), to be passed to parse_messages()."""
  # We use _convert_to_serializable so that we don't have to store field names.
  # TODO: Try protocol buffers and http://msgpack.org/ for tighter packing of
  #       records than cPickle.
  # cPickle.HIGHEST_PROTOCOL == 2.
  return zlib.compress(
      cPickle.dumps(map(_convert_to_serializable, messages), 2),
      COMPRESS_EFFORT)


def parse_messages(msgsdata):
  """Returns an iterable (currently a tuple) of parsed messages."""
  if not isinstance(msgsdata, str):
    raise TypeError
  s = zlib.decompress(msgsdata)
  if s.startswith('\x80\x02'):  # Pickle protocol 2.
    return tuple(map(_convert_from_serializable, cPickle.loads(s)))
  else:  # msgpack.
    # msgpack.pack({}) == '\x80'.
    # msgpack.pack({2: 3}) == '\x81\x02\x03'.
    # So there is no MsgPack serialized string starting with '\x80\x02'.
    return tuple(map(_convert_from_serializable, msgpack.unpack(s)))


def serialize_messages_msgpack(messages):
  return zlib.compress(msgpack.packb(map(_convert_to_serializable, messages)),
                       COMPRESS_EFFORT)


if USE_MSGPACK:
  derialize_messages = serialize_messages_msgpack


SYNC_INPUT_MESSAGE_FIELD_CHECKS = (
    ('address', 'string', None),
    ('body', 'string', None),
    ('date', 'integer', None),
    ('type', 'small integer', None),
    ('deleted', 'boolean', False),
    ('service_center', 'string', ''),
    ('error_code', 'small integer', 0),
    ('locked', 'small integer', 0),
    ('read', 'small integer', 0),
    ('seen', 'small integer', 0),
    ('status', 'small integer', -1),
    ('protocol', 'small integer', 0),
    ('conversation', 'string', ''),
)

MESSAGE_FIELDS = set(t[0] for t in SYNC_INPUT_MESSAGE_FIELD_CHECKS)
MESSAGE_FIELDS.add('phone')


def fix_sync_input_message(message):
  """Fixes message in place, returns the list of errors (in English)."""
  if not isinstance(message, dict):
    raise TypeError
  errors = []
  for field, expected_type, default in SYNC_INPUT_MESSAGE_FIELD_CHECKS:
    if field in message:
      value = message[field]
      if expected_type == 'string':
        if isinstance(value, str):
          message[field] = utf8_to_unicode(value)
        elif not isinstance(value, unicode):
          errors.append('Expected string in %s.' % field)
      elif expected_type == 'integer':
        if isinstance(value, (str, unicode)):
          try:
            value = int(value)
          except ValueError:
            value = None
        if not isinstance(value, (int, long)):
          errors.append('Expected integer in %s.' % field)
      elif expected_type == 'small integer':
        if isinstance(value, (str, unicode)):
          try:
            value = int(value)
          except ValueError:
            value = None
        if not isinstance(value, int) or not (-9999 <= value <= 9999):
          errors.append('Expected small integer in %s.' % field)
      elif expected_type == 'boolean':
        if isinstance(value, int):
          message[field] = bool(value)
        elif not isinstance(value, bool):
          errors.append('Expected boolean in %s.' % field)
    else:
      if default is None:
        errors.append('Missing %s.' % field)
      message[field] = default


  # Delete unknown fields.
  if len(message) > len(SYNC_INPUT_MESSAGE_FIELD_CHECKS):
    if len(message) > 3 * len(MESSAGE_FIELDS):
      message_copy = {}
      for field in MESSAGE_FIELDS:
        if field in message:
          message_copy[field] = message[field]
      message.clear()
      message.update(message_copy)
    else:
      for field in sorted(message):
        if field not in MESSAGE_FIELDS:
          del message[field]

  return errors


if __name__ == '__main__':
  import simplejson as json
  import time
  messages_json = open('webapp/smslong.txt').read()
  stats = {'jsonload_ms': 0, 'serialize_ms': 0, 'parse_ms': 0,
           'msgpack': int(USE_MSGPACK), 'compress': COMPRESS_EFFORT}
  ic = 50

  # compress:6 jsonload_ms:52 jsonsize:101852 msgcount:328 msgpack:1 parse_ms:27 serialize_ms:33 serializedsize:12907
  # compress:6 jsonload_ms:53 jsonsize:101852 msgcount:328 msgpack:0 parse_ms:6 serialize_ms:10 serializedsize:16906
  # compress:3 jsonload_ms:52 jsonsize:101852 msgcount:328 msgpack:0 parse_ms:6 serialize_ms:9 serializedsize:18182
  # compress:9 jsonload_ms:52 jsonsize:101852 msgcount:328 msgpack:0 parse_ms:6 serialize_ms:11 serializedsize:16814
  stats['jsonsize'] = len(messages_json)
  now = time.time()
  for iteration in xrange(ic):
    messages = json.loads(messages_json)['messages']
  #assert 0, (len(msgpack.dumps(messages)), stats['jsonsize'])
  stats['jsonload_ms'] += time.time() - now
  stats['msgcount'] = len(messages)
  now = time.time()
  for iteration in xrange(ic):
    msgsdata = serialize_messages(messages)
  stats['serialize_ms'] += time.time() - now
  stats['serializedsize'] = len(msgsdata)
  now = time.time()
  for iteration in xrange(ic):
    messages2 = parse_messages(msgsdata)
  stats['parse_ms'] += time.time() - now
  for name in ('jsonload_ms', 'serialize_ms', 'parse_ms'):
    stats[name] = int(stats[name] * 1000 / ic + .5)
  print ' '.join('%s:%s' % (name, stats[name]) for name in sorted(stats))
  
  assert len(messages2) == len(messages), len(messages2)
  for i in xrange(len(messages)):
    m1 = dict(messages[i])
    m1.pop('conversation', None)
    m1.setdefault('deleted', False)
    m1.setdefault('phone', u'ap')
    m1.setdefault('protocol', 0)
    m1.setdefault('service_center', u'')
    m2 = dict(messages2[i])
    del m2['conversation']
    #assert len(m1) == len(m2), (len(m1), len(m2))
    for name in sorted(m2):
      # Converts name between unicode and str automatically.
      assert m1.get(name) == m2.get(name), (i, name, m1.get(name), m2.get(name))
